from codesecurity.tasks.crawler import NpmCrawler,NpmCrawlerMeta
from codesecurity.tasks.crawler.npm_crawler import NpmMetaDB
from codesecurity.data.api import *
from codesecurity.tasks.common import code_api
from codesecurity.utils.pretty_print import lazy_print
from codesecurity.data.history import HistoryFile

from collections import Counter, OrderedDict

import fire
import os
import datetime

default_dir='test_data/20231220_20231227_packages_npm'
list_file='test_data/package_between_20231220_20231227.txt'

def read_list_file(list_file):
    with open(list_file,'r') as f:
        lines=f.readlines()
        lines=[e.split()[0] for e in lines]
        
        ret=OrderedDict()
        for line in lines:
             # find_last @
            package_name_index=line.find('@',1)
            package_name=line[:package_name_index]
            version=line[package_name_index+1:]
            
            if package_name not in ret:
                ret[package_name]=''
            
            if version>ret[package_name]:
                ret[package_name]=version
                

        return ret.items()

def preprocess():
    meta=NpmCrawlerMeta.default(default_dir)
    crawler=NpmCrawler(meta)
    crawler.preprocess_packages()

def crawl_tgz(base_dir,package_list_file,history_file=None):
    
    if history_file is None:
        package_list_name=os.path.basename(package_list_file)
        package_list_name=os.path.splitext(package_list_name)[0]
        history_file=f'{base_dir}/{package_list_name}_history.txt'
    
    meta=NpmCrawlerMeta.default(base_dir)
    crawler=NpmCrawler(meta)
    
    
    
    error_packages=crawler.crawl_tgzs(read_list_file(package_list_file),history_file)
    
    if len(error_packages)==0:
        print('All packages are crawled successfully.')
        if os.remove(history_file):
            print(f'{history_file} is removed.')
        
    package_list_name=os.path.basename(package_list_file)
    with open(f'exp_result/{package_list_name}.error','w') as f:
        for error_package in error_packages:
            f.write(f'{error_package}\n')

def preprocess_tgzs(base_dir):
    meta=NpmCrawlerMeta.default(base_dir)
    crawler=NpmCrawler(meta)
    crawler.preprocess_tgzs(remove_origin=True)

def generate_package_list(start_time,end_time,exclude_strs=''):
    
    exclude_strs=exclude_strs.split(',')
    
    start_time_str=start_time.replace('-','')
    end_time_str=end_time.replace('-','')
    
    list_file=f'test_data/package_between_{start_time_str}_{end_time_str}.txt'
    npm_db=NpmMetaDB.default()    
    count=-1
    search_number=0
    with open(list_file,'w') as f:
        for meta in npm_db.lastest_docs(start_time,end_time):
            for exclude_str in exclude_strs:
                if exclude_str in meta.package_name:
                    continue
            search_number+=1
            if meta.time_meta is None:
                print(f'{meta.package_name} has no time meta') 
                continue

            f.write(f'{meta.package_name}@{meta.latest_version} {meta.time_meta.modified}\n')
            count+=1
            
            if search_number%1000==0 and search_number>1:
                print(f'In {search_number} packages, find {count} in range versions.')

def generate_hot_package_list():
    list_file='test_data/hot_package_list.txt'
    metas=NpmCrawler.get_hot_package_list()
    with open(list_file,'w') as f:
        for meta in metas:
            f.write(f'{meta.package_name}@{meta.version}\n')
            print(f'{meta.package_name}@{meta.version}\n')

def generate_hot_api_list(target_dir,label):
    list_file='exp_result/hot_api_{}_{}_list.txt'
    list_file=list_file.format(label,'{}{}{}'.format(datetime.datetime.now().year,datetime.datetime.now().month,datetime.datetime.now().day))
    
    threshold=25
    limit=1000000
    
    
    files=iter_dir(target_dir)
    
    counter=Counter()
    
    lazy_print('building api table...',1000,restore=True)
    for i,file in enumerate(files):
        apis=code_api.get_api_list(file)
        counter.update(apis.keys())
        lazy_print(f'processed file number {i}. {file}.',1000)
        
        if i>limit:
            break
        
    for key in counter.most_common(100):
        print(key)
        
    with open(list_file,'w') as f:
        for key in counter.most_common():
            if key[1]<threshold:
                break
            
            content=key[0].replace('\n','')
            
            
            f.write(f'{content},{key[1]}\n')
    




if __name__=="__main__":
    fire.Fire()